import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torchtext
import numpy as np

# 测试资料
word_to_ix = {"hello": 0, "world": 1}
# 词汇表(vocabulary)含2个单字, 转换为5维的向量
embeds = nn.Embedding(2, 5)
# 测试 hello
lookup_tensor = torch.LongTensor([word_to_ix["hello"]])
hello_embed = embeds(lookup_tensor)
print(hello_embed)


from torchtext.data.utils import get_tokenizer

tokenizer = get_tokenizer('basic_english')

text = 'Could have done better.'
print(tokenizer(text))

from torchtext.vocab import vocab
from collections import Counter, OrderedDict

# BOW 统计
counter = Counter(tokenizer(text))
# 依出现次数降幂排列
sorted_by_freq_tuples = sorted(counter.items(), key=lambda x: x[1], reverse=True)
# 建立词汇字典
ordered_dict = OrderedDict(sorted_by_freq_tuples)

# 建立词汇表物件，并加一个未知单字(unknown)的索引值
vocab_object = torchtext.vocab.vocab(ordered_dict, specials=["<unk>"])
# 设定词汇表预设值为未知单字(unknown)的索引值
vocab_object.set_default_index(vocab_object["<unk>"])

# 测试
vocab_object['done']